//============================================================================
// Name        : ManagerBerthier.cpp
// Author      : 
// Version     :
// Copyright   : Your copyright notice
// Description : Hello World in C++, Ansi-style
//============================================================================

#include <iostream>
#include "ManagerBerthier.h"
#include <ext/hash_map>
#include <list>
#include <vector>
#include <queue>
#include "../classes_net/URLFileDealer.h"
#include "../classes_net/DownloadParser.h"
#include "Hasher.h"
#include<string>
#include <boost/tr1/memory.hpp>
#include <boost/thread/mutex.hpp>
#include "../classes_net/URL.h"
#include "../classes_outros/DownloaderQueueException.h"
#include "../classes_outros/Config.h"
#include "../classes_outros/RobotsDownloadInterface.h"
#include "ThreadFunctionDatabase.h"

using namespace std;
using namespace __gnu_cxx;

std::tr1::shared_ptr<ManagerBerthier> ManagerBerthier::theManagerBerthier(new ManagerBerthier());
const unsigned int ManagerBerthier::URLS_PER_QUEUE = Config::getUrlsPerQueue();

ManagerBerthier::ManagerBerthier() 
:timeBetweenVisits(Config::getTimeBetweenVisits()),
 queuesInitialized(false)
{
}



void ManagerBerthier::initializeDownloaderQueues (const int quantity) throw(DownloaderQueueException){
	if (queuesInitialized) {
		DownloaderQueueException dqe("As filas jah foram inicializadas");
		throw dqe;
	}
	urlQueueVector.reserve(quantity);
	for (int i = 0; i<quantity; i++)
		urlQueueVector.push_back(queue<URL *>() );	
	cout << "mb::init TAMANHO urlQueueVector " << urlQueueVector.size() << endl; 
	downloaderQueueMutex = new boost::mutex[quantity];
	numQueues = quantity;
	queuesInitialized = true;
}

bool ManagerBerthier::refillQueuesOnce() throw(DownloaderQueueException){
	if (!queuesInitialized){
		DownloaderQueueException dqe("MB::refillQueuesOnce chamado antes de inicializar vetor");
		throw dqe;
	}
	
	bool conseguiu = false;
	
	{
	boost::mutex::scoped_lock urlQueueLock(urlQueueBufferMutex);
	if (urlQueueBuffer.size() < numQueues)
		refillQueueBuffer();
	}
	vector< queue<URL *> >::iterator it;
//	cout << "\n\nMEIO refillQueuesOnce " << urlQueueBuffer.size() << endl << endl;
//	cout << "\n\nMEIO refillQueuesOnce2 " << urlQueueVector.size() << endl << endl;

	
	int i = 0;
	URL * url;
	bool hasNewUrl = false;
	for (it = urlQueueVector.begin(); (it!=urlQueueVector.end()) && !urlQueueBuffer.empty(); it++) {
		//cout << "loop: " << it->size() << " " << urlQueueBuffer.size() << endl; 
		if (!hasNewUrl)
			url = findURL();
		
		boost::mutex::scoped_lock downloaderQueueLock(downloaderQueueMutex[i++]);
		if (it->size() <= URLS_PER_QUEUE){
			//cout << "Inserindo na fila " << i-1 << ": " << url->toString() << endl;
			it->push(url);
			hasNewUrl = false;
			conseguiu = true;
		}
		else if (url != NULL) {
			//cout << "ENCONTREI URL NOVA: " << url->toString() << endl; 
			hasNewUrl = true;
		}
		else {
//			cout << "ENCONTREI URL NULL" << endl; 
		}
	}
	return conseguiu;
}

URL * ManagerBerthier::getNewURL(int id) throw(DownloaderQueueException){
	//cout << "\n\n\n\n AAAAAAAAAAAAAAAAAAAAAAAAAAAA" << endl;
	
	if (!queuesInitialized){
		DownloaderQueueException dqe ("As filas dos downloaders n�o foram inicializadas");
		throw dqe;
	}
	
	URL * url;
	boost::mutex::scoped_lock downloaderQueueLock(downloaderQueueMutex[id]);
	if (urlQueueVector[id].empty()) {
		//cout << "RETORNANDO NULL mb::getNewUrl: " << urlQueueVector[id].size() << endl;
		return NULL;
	}
	else {
//		cout << "mb::getNew-Tirando elemento da fila " << id << ", de tamanho " << urlQueueVector[id].size() << endl;
		url = urlQueueVector[id].front();
		urlQueueVector[id].pop();
		//cout << "\nVETOR DO MC[" << id << "]: " << url->toString() << endl;
		return url;
	}	
}


string ManagerBerthier::getRobots(string domain) {
	DownloadParser downloader;
//	cout << "Puxando o robots.txt do dominio " << domain << endl;
	downloader.setURL(domain);
	return downloader.getRobot();
}

/**
 * Verificar se a url � v�lida qdo obter!!!
 */
URL * ManagerBerthier::findURL() {
	URL * url;
	time_t timeMc;
	std::tr1::shared_ptr<URLFileDealer> urlFileDealer = URLFileDealer::getInstance();
	
	//Ele bloqueia na inicializacao e desbloqueia no destrutor (nao preciso me preocupar)
	while (true) {
		{
			boost::mutex::scoped_lock urlQueueLock(urlQueueBufferMutex);
			if (urlQueueBuffer.empty()) {
				bool conseguiu = refillQueueBuffer();
				if (!conseguiu) {
					//cout << "mb:find-RETORNANDO NULL" << endl;
					return NULL; 
				}
			}
				
			
			url = urlQueueBuffer.front(); 
			
			//cout << "url: " << url.toString() << endl;
			urlQueueBuffer.pop();	
		}
		uint domainHash = Hasher::simpleHash( url->get_dominio() );
		//Se o dom�nio n�o existir, insiro no hash de dominios, e retorno a url
		
		boost::mutex::scoped_lock domainHashMapLock(domainHashMapMutex);
		if(domainHashMap[domainHash] == NULL){
			//cout << "1" << endl;
			
			
			
			//Verifica numero maximo de numThreadsRobots, se maior, retorna url para disco e continua
//			if (ThreadFunctionDatabase::getNumThreadsRobots() >= Config::getMaxRobotsThreads()){
//				cout << "ManagerBerthier::findURL ------->   getNumThreadsRobots() >= maximo " << endl; 
//				urlFileDealer->addToDiskBuffer(url);
//				continue;
//			}			
			
			
			//domainHashMapLock.unlock();
//			string robots = getRobots(url->get_dominio());
//			cout << "DOM:  " << url.get_dominio() << endl;
//			cout << "ROBOTS:" << endl << robots << endl;
			time(&timeMc);
			Dominio * dominio = new Dominio(url->get_dominio(), /*robots,*/ timeMc - timeBetweenVisits);
			//boost::mutex::scoped_lock domainHashMapLock2(domainHashMapMutex);
			
			
//			ThreadFunctionDatabase::incNumThreadsRobots();
//			
//			pthread_attr_t attr;
//			size_t stacksize;
//			pthread_t robotThread;
//			int rc;
//			pthread_attr_init(&attr);
//			stacksize = Config::getRobotsThreadStackSize();
//			pthread_attr_setstacksize (&attr, stacksize);
//			rc = pthread_create(&robotThread, &attr, ThreadFunctionDatabase::getDomainRobots, (void *)dominio); 
//	    	if (rc){
//	    	  cout << "MB::findURL - ERROR1; return ROBOTS code from pthread_create() is " << rc << "\n";
//	    	  exit(-1);
//	    	}
			if (!RobotsDownloadInterface::put(dominio)) {
				delete dominio;
				urlFileDealer->addToDiskBuffer(url);
		    	domainHashMapLock.unlock();
				continue;
			}
			
			domainHashMap[domainHash] = dominio;
	    	domainHashMapLock.unlock();
			
			urlFileDealer->addToDiskBuffer(url);
		}
		//se o dominio j� existir, verifico o tempo e permiss�o do robots
		else {
			//cout << "2" << endl;
			time(&timeMc);
			Dominio * dominio = domainHashMap[domainHash];
			
			domainHashMapLock.unlock();
			if ( !dominio->isRobotsValid() ) {
				urlFileDealer->addToDiskBuffer(url);
				continue;
			}
			
			if (!dominio->isAllowedByRobots(url)) {
				//cout << "3" << endl;
				//url.becomeInvalid();
				delete url;
				continue;
			}
			else if ( ( timeMc - dominio->getTimestamp() ) < timeBetweenVisits) {
				//cout << "4" << endl;
				urlFileDealer->addToDiskBuffer(url);
				//url.becomeInvalid();
				continue;
			}
			else {
				//cout << "5" << endl;
				dominio->updateTimestamp(timeMc);
				//cout << "mb:find-RETORNANDO:" << url->toString() << endl;
				return url;
			}
		}
	}
}




//Inidica HashDomain do disco
void ManagerBerthier::initHashDomain(URL url){
	
	time_t timeMc;
	uint domainHash = Hasher::simpleHash( url.get_dominio() );
	
	//Se o dominio nao existir, insiro no hash de dominios
	boost::mutex::scoped_lock domainHashMapLock(domainHashMapMutex);
	
	if(domainHashMap[domainHash] == NULL){
		time(&timeMc);
		Dominio * dominio = new Dominio(url.get_dominio(), /*robots,*/ timeMc - timeBetweenVisits);

		if (!RobotsDownloadInterface::put(dominio)) {
			delete dominio;
	    	domainHashMapLock.unlock();
			return;
		}
		
		cout << "Iniciando Dominio:  " << url.get_dominio() << endl;

		domainHashMap[domainHash] = dominio;
    	domainHashMapLock.unlock();
	}

	return;
}




bool ManagerBerthier::refillQueueBuffer() {
	std::tr1::shared_ptr<URLFileDealer> theURLFileDealer = URLFileDealer::getInstance();
	list<URL *> urlList = theURLFileDealer->getNewURLs();
	if (urlList.size() == 0)
		return false;
	list<URL *>::const_iterator it;
	
	for (it = urlList.begin(); it!=urlList.end(); it++) {
		urlQueueBuffer.push(*it);
	}
//	cout << "MC tb suga pelo menos " << urlQueue.size() << " pessoa por dia" << endl;
	return true;	
}

//ManagerBerthier &ManagerBerthier::getInstance()
//{
//	//tomar cuidado nesse if qdo for multithread!!!
//    if(theManagerBerthier.get() == 0)
//        theManagerBerthier.reset(new ManagerBerthier);
//    return *theManagerBerthier;
//}

/*void ManagerBerthier::testPriorityQueue() {
	URL * url;
	url = new URL("www.uol.com.br/", "pasta1/pasta2/arquivo.html", "http://");
	urlQueue.push(url);
	url = new URL("www.uol.com.br/", "pasta3/outroArquivo.htm", "http://");
	urlQueue.push(url);
	url = new URL("www.uol.com.br/", "pasta4/pasta5/pasta6/prof3.htm", "http://");
	urlQueue.push(url);
	url = new URL("www.uol.com.br/", "index.html", "http://");
	urlQueue.push(url);
	url = new URL("www.google.com/", "", "http://");
	urlQueue.push(url);
	url = new URL("www.google.com/", "prof1/atchim.html", "http://");
	urlQueue.push(url);
	url = new URL("www.google.com/", "abc.html", "http://");
	urlQueue.push(url);
	url = new URL("www.google.com/", "prof2/prof2/prof.html", "http://");
	urlQueue.push(url);
	
	while (!urlQueue.empty()) {
		url = urlQueue.top();
		cout << url->toString() << endl;
		cout << "Profundidade: " << url->getDepth() << endl;
		urlQueue.pop();
		delete url;
	}
	cout << "Saih do for" << endl;
}*/
